# 改写课程的代码,爬取东方财富龙虎榜股票信息存入json
# 编写时间 2021/05/
import requests
from bs4 import BeautifulSoup

def getHTMLText(url, code="utf-8"):     # 获取HTML代码[数据存储在静态网页中]

    headers = {
        'Connection': 'keep-alive',
        'Cache-Control': 'max-age=0',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        }
    try:
        r = requests.get(url, headers=headers, verify=False)
        r.raise_for_status()
        r.encoding = code
        return r.text
    except:
        return ""

def getStockList(lst, stockURL):                # 获取股票列表
    html = getHTMLText(stockURL,code="gb2312")      # 龙虎榜数据存储在静态网页中,按照"gb2312"编码
    soup = BeautifulSoup(html, 'html.parser')    
    a = soup.find_all('span',class_="wname")
    for i in a:
        try:
            stockid = i.find('a')['data_code']
            print(stockid)
            lst.append(stockid)
        except:
            continue

def getStockInfo(lst, stockURL, fpath):     # 获取股票信息.现在一般都存在XHR文件中,不在HTML文件
    count = 0
    for stock in lst:
        url = stockURL + stock + "/"
        html = getHTMLText(url)         # 同花顺一部分信息存在html
        try:
            if html=="":
                continue
            infoDict = {}
            soup = BeautifulSoup(html, 'html.parser')

            name = soup.find('strong',attrs={'id':'stockNamePlace'}).text
            infoDict.update({'股票名称': name})

            stockInfo = soup.find('dl',class_="company_details")
            keyList = stockInfo.find_all('dt')
            valueList = stockInfo.find_all('dd')
            del valueList[2]
            for i in reversed(range(len(keyList))):
                if valueList[i].attrs == {}:
                    val = valueList[i].text
                else :
                    val = valueList[i].attrs['title']
                key = keyList[i].text
                infoDict[key] = val     #主营业务有两个'dd',所以错位了
            
            with open(fpath, 'a', encoding='utf-8') as f:
                f.write( str(infoDict) + '\n' )
                count = count + 1
                print("\r当前进度: {:.2f}%".format(count*100/len(lst)),end="")
        except:
            count = count + 1
            print("\r当前进度: {:.2f}%".format(count*100/len(lst)),end="")  # /r可更新当前print,非滚动
            continue

def main():
    stock_list_url = 'http://data.eastmoney.com/stock/lhb.html'
    stock_info_url = 'http://stockpage.10jqka.com.cn/'
    output_file = r'Learning Record\Learning pro\TongHuaShunStockInfo.json'
    slist=[]
    getStockList(slist, stock_list_url)
    getStockInfo(slist, stock_info_url, output_file)


main()
print('hello stock')